from scrapy.http import Request, FormRequest, HtmlResponse

import gtk
import webkit
import jswebkit

class JSMiddleware(object):

    def process_spider_input(slef, response, spider):
        if(type(response) is not FormRequest):
            webview = webkit.WebView()
            webview.connect( 'load-finished', lambda v,f: gtk.main_quit() )
            webview.load_uri(response.url)
            gtk.main()
            js = jswebkit.JSContext( webview.get_main_frame().get_global_context() )
            renderedBody = str( js.EvaluateScript('document.documentElement.innerHTML'))
            return HtmlResponse( response.url,body=renderedBody)

